From 3bc8ab91a2694380fc7c46d2b53351be8e6f12f3 Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Sat, 13 Apr 2019 14:11:30 +0100 Subject: [PATCH] ci: Add an HTML report generator MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit The JUnit cover report is useful, but only up to a point; for instance, it's not used unless it's part of a merge request. This means you don't get a report if you're pushing to a branch that does not have an MR open. With a simple Python script and some minimal templating, we can generate an HTML report from the "I Can't Believe it's not JSON™" log that Meson produces, and keep it as a CI artifact. --- .gitlab-ci.yml | 1 + .gitlab-ci/Dockerfile | 2 + .gitlab-ci/meson-html-report.py | 164 ++++++++++++++++++++++++++++++++ .gitlab-ci/test-docker.sh | 8 +- 4 files changed, 174 insertions(+), 1 deletion(-) create mode 100755 .gitlab-ci/meson-html-report.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 57cb4ab9c1..6528a6f149 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,6 +26,7 @@ fedora-x86_64: paths: - "${CI_PROJECT_DIR}/_build/meson-logs" - "${CI_PROJECT_DIR}/_build/report.xml" + - "${CI_PROJECT_DIR}/_build/report.html" - "${CI_PROJECT_DIR}/_build/testsuite/reftests/output/*.png" cache: key: "$CI_JOB_NAME" diff --git a/.gitlab-ci/Dockerfile b/.gitlab-ci/Dockerfile index 8c32e8fa5b..57b1983fda 100644 --- a/.gitlab-ci/Dockerfile +++ b/.gitlab-ci/Dockerfile @@ -73,6 +73,8 @@ RUN dnf -y install \ RUN pip3 install meson==0.50.0 +RUN pip3 install jinja2 + ARG HOST_USER_ID=5555 ENV HOST_USER_ID ${HOST_USER_ID} RUN useradd -u $HOST_USER_ID -ms /bin/bash user diff --git a/.gitlab-ci/meson-html-report.py b/.gitlab-ci/meson-html-report.py new file mode 100755 index 0000000000..a5d1d82409 --- /dev/null +++ b/.gitlab-ci/meson-html-report.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 + +# Copyright 2019 GNOME Foundation + +# Turns a test log generated by Meson into an HTML report + +import argparse +import datetime +import json +import os +import sys +from jinja2 import Template + +REPORT_TEMPLATE = ''' + + + + {{ report.project_name }} Test Report + + + +
+

{{ report.project_name }} :: Test Reports

+
+

Branch: {{ report.branch_name }}

+

Date:

+ {% if report.job_id %}

Job ID: {{ report.job_id }}

{% endif %} +
+
+ +
+
+
+

Summary

+
    +
  • Total units: {{ report.total_units }}
    • +
  • Passed: {{ report.total_successes }}
    • +
  • Failed: {{ report.total_failures }}
    • + +
+
+ + {% for suite_result in report.results_list %} +
+
+

Suite: {{ suite_result.suite_name }}

+
    +
  • Units: {{ suite_result.n_units }}
    • +
  • Passed: {{ suite_result.n_successes }}
    • +
  • Failed: {{ suite_result.n_failures }}
    • +
+ {% for failure in suite_result.failures %} + {% if loop.first %} +
+

Failures

+
    + {% endif %} +
  • {{ failure.name }} - result: {{ failure.result }}
    + 
    {{ failure.stdout }}
    +
    • + {% if loop.last %} +
+
+ {% endif %} + {% endfor %} +
+ {% endfor %} + +
+ + +''' + +aparser = argparse.ArgumentParser(description='Turns a Meson test log into an HTML report') +aparser.add_argument('--project-name', metavar='NAME', + help='The project name', + default='Unknown') +aparser.add_argument('--job-id', metavar='ID', + help='The job ID for the report', + default=None) +aparser.add_argument('--branch', metavar='NAME', + help='Branch of the project being tested', + default='master') +aparser.add_argument('--output', metavar='FILE', + help='The output HTML file, stdout by default', + type=argparse.FileType('w', encoding='UTF-8'), + default=sys.stdout) +aparser.add_argument('infile', metavar='FILE', + help='The input testlog.json, stdin by default', + type=argparse.FileType('r', encoding='UTF-8'), + default=sys.stdin) + +args = aparser.parse_args() + +outfile = args.output + +suites = {} +for line in args.infile: + data = json.loads(line) + (full_suite, unit_name) = data['name'].split(' / ') + (project_name, suite_name) = full_suite.split(':') + + unit = { + 'project-name': project_name, + 'suite': suite_name, + 'name': unit_name, + 'duration': data['duration'], + 'returncode': data['returncode'], + 'result': data['result'], + 'stdout': data['stdout'], + } + + units = suites.setdefault(full_suite, []) + units.append(unit) + +report = {} +report['date'] = datetime.datetime.utcnow() +report['locale_date'] = report['date'].strftime("%c") +report['project_name'] = args.project_name +report['job_id'] = args.job_id +report['branch_name'] = args.branch +report['total_successes'] = 0 +report['total_failures'] = 0 +report['total_units'] = 0 +report['results_list'] = [] + +for name, units in suites.items(): + (project_name, suite_name) = name.split(':') + print('Processing {} suite {}:'.format(project_name, suite_name)) + + def if_failed(unit): + if unit['result'] in ['FAIL', 'TIMEOUT']: + return True + return False + + def if_succeded(unit): + if unit['result'] in ['OK', 'EXPECTEDFAIL', 'SKIP']: + return True + return False + + successes = list(filter(if_succeded, units)) + failures = list(filter(if_failed, units)) + + n_units = len(units) + n_successes = len(successes) + n_failures = len(failures) + + report['total_units'] += n_units + report['total_successes'] += n_successes + report['total_failures'] += n_failures + print(' - {}: {} total, {} pass, {} fail'.format(suite_name, n_units, n_successes, n_failures)) + + suite_report = { + 'suite_name': suite_name, + 'n_units': n_units, + 'successes': successes, + 'n_successes': n_successes, + 'failures': failures, + 'n_failures': n_failures, + } + report['results_list'].append(suite_report) + +template = Template(REPORT_TEMPLATE) +outfile.write(template.render({'report': report})) diff --git a/.gitlab-ci/test-docker.sh b/.gitlab-ci/test-docker.sh index 95ed71edbd..45de67000a 100755 --- a/.gitlab-ci/test-docker.sh +++ b/.gitlab-ci/test-docker.sh @@ -37,11 +37,17 @@ xvfb-run -a -s "-screen 0 1024x768x24" \ # Save the exit code exit_code=$? -# We always want to run the report generator +# We always want to run the report generators $srcdir/.gitlab-ci/meson-junit-report.py \ --project-name=gtk \ --job-id="${CI_JOB_NAME}" \ --output=report.xml \ meson-logs/testlog.json +$srcdir/.gitlab-ci/meson-html-report.py \ + --project-name=GTK \ + --job-id="${CI_JOB_NAME}" \ + --output=report.html \ + meson-logs/testlog.json + exit $exit_code -- 2.30.2